{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "781b40ff",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-06T07:42:55.366741Z",
     "iopub.status.busy": "2024-10-06T07:42:55.365993Z",
     "iopub.status.idle": "2024-10-06T07:43:08.460126Z",
     "shell.execute_reply": "2024-10-06T07:43:08.458932Z"
    },
    "papermill": {
     "duration": 13.101367,
     "end_time": "2024-10-06T07:43:08.462850",
     "exception": false,
     "start_time": "2024-10-06T07:42:55.361483",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install optuna-integration[xgboost] -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b0263af6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-06T07:43:08.470287Z",
     "iopub.status.busy": "2024-10-06T07:43:08.469563Z",
     "iopub.status.idle": "2024-10-06T07:43:11.761614Z",
     "shell.execute_reply": "2024-10-06T07:43:11.760625Z"
    },
    "papermill": {
     "duration": 3.298197,
     "end_time": "2024-10-06T07:43:11.763949",
     "exception": false,
     "start_time": "2024-10-06T07:43:08.465752",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn import preprocessing\n",
    "from sklearn.metrics import root_mean_squared_error\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import OrdinalEncoder\n",
    "from xgboost import XGBRegressor\n",
    "import cupy as cp\n",
    "import optuna"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9728e1e1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-06T07:43:11.770799Z",
     "iopub.status.busy": "2024-10-06T07:43:11.770032Z",
     "iopub.status.idle": "2024-10-06T07:43:15.920001Z",
     "shell.execute_reply": "2024-10-06T07:43:15.918924Z"
    },
    "papermill": {
     "duration": 4.155856,
     "end_time": "2024-10-06T07:43:15.922392",
     "exception": false,
     "start_time": "2024-10-06T07:43:11.766536",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Loading data \n",
    "X_train = pd.read_csv(\"../input/30-days-of-ml/train.csv\").iloc[:100_000, :]\n",
    "X_test = pd.read_csv(\"../input/30-days-of-ml/test.csv\")\n",
    "\n",
    "# Preparing data as a tabular matrix\n",
    "y_train = X_train.target\n",
    "X_train = X_train.set_index('id').drop('target', axis='columns')\n",
    "X_test = X_test.set_index('id')\n",
    "\n",
    "# Pointing out categorical features\n",
    "categoricals = [item for item in X_train.columns if 'cat' in item]\n",
    "\n",
    "# Dealing with categorical data using OrdinalEncoder\n",
    "ordinal_encoder = OrdinalEncoder()\n",
    "X_train[categoricals] = ordinal_encoder.fit_transform(X_train[categoricals])\n",
    "X_test[categoricals] = ordinal_encoder.transform(X_test[categoricals])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8a948470",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-06T07:43:15.929229Z",
     "iopub.status.busy": "2024-10-06T07:43:15.928858Z",
     "iopub.status.idle": "2024-10-06T07:43:15.940119Z",
     "shell.execute_reply": "2024-10-06T07:43:15.939363Z"
    },
    "papermill": {
     "duration": 0.016646,
     "end_time": "2024-10-06T07:43:15.941924",
     "exception": false,
     "start_time": "2024-10-06T07:43:15.925278",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from optuna_integration import XGBoostPruningCallback\n",
    "\n",
    "def objective(trial):\n",
    "    \n",
    "    params = {\n",
    "            'learning_rate': trial.suggest_float(\"learning_rate\", 0.01, 1.0, log=True),\n",
    "            'reg_lambda': trial.suggest_float(\"reg_lambda\", 1e-9, 100.0, log=True),\n",
    "            'reg_alpha': trial.suggest_float(\"reg_alpha\", 1e-9, 100.0, log=True),\n",
    "            'subsample': trial.suggest_float(\"subsample\", 0.1, 1.0),\n",
    "            'colsample_bytree': trial.suggest_float(\"colsample_bytree\", 0.1, 1.0),\n",
    "            'max_depth': trial.suggest_int(\"max_depth\", 1, 7),\n",
    "            'min_child_weight': trial.suggest_int(\"min_child_weight\", 1, 7),\n",
    "            'gamma': trial.suggest_float(\"gamma\", 0.1, 1.0, step=0.1)\n",
    "    }\n",
    "\n",
    "    model = XGBRegressor(\n",
    "        random_state=0,\n",
    "        tree_method = \"hist\", \n",
    "        device = \"gpu\",\n",
    "        n_estimators=10_000,\n",
    "        early_stopping_rounds=300,\n",
    "        callbacks=[XGBoostPruningCallback(trial, 'validation_0-rmse')],\n",
    "        **params,\n",
    "    )\n",
    "    \n",
    "    model.fit(x, y, eval_set=[(x_val, y_val)], verbose=1000)\n",
    "    preds = model.predict(cp.asarray(x_test.values))\n",
    "    rmse = root_mean_squared_error(y_test, preds)\n",
    "    return rmse\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "abfa3e0d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-06T07:43:15.947947Z",
     "iopub.status.busy": "2024-10-06T07:43:15.947580Z",
     "iopub.status.idle": "2024-10-06T07:44:22.035147Z",
     "shell.execute_reply": "2024-10-06T07:44:22.034232Z"
    },
    "papermill": {
     "duration": 66.092761,
     "end_time": "2024-10-06T07:44:22.037175",
     "exception": false,
     "start_time": "2024-10-06T07:43:15.944414",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:43:16,000] A new study created in memory with name: no-name-97bf573b-29a9-4036-b9ca-41a9de8d7ab7\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74298\n",
      "[1000]\tvalidation_0-rmse:0.72725\n",
      "[2000]\tvalidation_0-rmse:0.72434\n",
      "[3000]\tvalidation_0-rmse:0.72258\n",
      "[4000]\tvalidation_0-rmse:0.72187\n",
      "[5000]\tvalidation_0-rmse:0.72121\n",
      "[6000]\tvalidation_0-rmse:0.72081\n",
      "[7000]\tvalidation_0-rmse:0.72070\n",
      "[7083]\tvalidation_0-rmse:0.72070\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:43:30,261] Trial 0 finished with value: 0.7202722001387207 and parameters: {'learning_rate': 0.029142577621650317, 'reg_lambda': 0.006548888217760044, 'reg_alpha': 8.725393704306269, 'subsample': 0.2637717396252206, 'colsample_bytree': 0.10480925898128796, 'max_depth': 2, 'min_child_weight': 4, 'gamma': 0.1}. Best is trial 0 with value: 0.7202722001387207.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74278\n",
      "[1000]\tvalidation_0-rmse:0.72893\n",
      "[2000]\tvalidation_0-rmse:0.72707\n",
      "[3000]\tvalidation_0-rmse:0.72583\n",
      "[4000]\tvalidation_0-rmse:0.72489\n",
      "[5000]\tvalidation_0-rmse:0.72416\n",
      "[6000]\tvalidation_0-rmse:0.72354\n",
      "[7000]\tvalidation_0-rmse:0.72305\n",
      "[8000]\tvalidation_0-rmse:0.72268\n",
      "[9000]\tvalidation_0-rmse:0.72237\n",
      "[9999]\tvalidation_0-rmse:0.72207\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:43:47,422] Trial 1 finished with value: 0.7221858632495973 and parameters: {'learning_rate': 0.05064403148512496, 'reg_lambda': 0.00037714890135213974, 'reg_alpha': 41.69327086483488, 'subsample': 0.675798981460365, 'colsample_bytree': 0.9214199806641051, 'max_depth': 1, 'min_child_weight': 3, 'gamma': 0.30000000000000004}. Best is trial 0 with value: 0.7202722001387207.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74004\n",
      "[330]\tvalidation_0-rmse:0.72695\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:43:48,153] Trial 2 finished with value: 0.7251112259421767 and parameters: {'learning_rate': 0.5383916927294936, 'reg_lambda': 0.011096605102822064, 'reg_alpha': 53.75136396143103, 'subsample': 0.9951118036556551, 'colsample_bytree': 0.571911141225583, 'max_depth': 6, 'min_child_weight': 7, 'gamma': 0.8}. Best is trial 0 with value: 0.7202722001387207.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74043\n",
      "[307]\tvalidation_0-rmse:0.82546\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:43:49,139] Trial 3 finished with value: 0.7358478289770027 and parameters: {'learning_rate': 0.23932229907406652, 'reg_lambda': 0.1894593396224293, 'reg_alpha': 5.018720695905843e-09, 'subsample': 0.12085636066141688, 'colsample_bytree': 0.9546951278453969, 'max_depth': 5, 'min_child_weight': 6, 'gamma': 0.1}. Best is trial 0 with value: 0.7202722001387207.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74185\n",
      "[342]\tvalidation_0-rmse:0.74770\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:43:50,399] Trial 4 finished with value: 0.72820137511862 and parameters: {'learning_rate': 0.39119339191771757, 'reg_lambda': 0.0450802432334948, 'reg_alpha': 17.19296171190493, 'subsample': 0.43159001005847986, 'colsample_bytree': 0.3186957061201342, 'max_depth': 6, 'min_child_weight': 4, 'gamma': 0.4}. Best is trial 0 with value: 0.7202722001387207.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.73989\n",
      "[332]\tvalidation_0-rmse:0.75036\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:43:51,460] Trial 5 finished with value: 0.7305090415633273 and parameters: {'learning_rate': 0.28944494766743833, 'reg_lambda': 9.791455873535009e-06, 'reg_alpha': 0.10471659553793466, 'subsample': 0.44217306475195883, 'colsample_bytree': 0.9426410266558707, 'max_depth': 4, 'min_child_weight': 1, 'gamma': 0.6}. Best is trial 0 with value: 0.7202722001387207.\n",
      "[I 2024-10-06 07:43:51,605] Trial 6 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74222\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:43:51,744] Trial 7 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74253\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:43:51,885] Trial 8 pruned. Trial was pruned at iteration 0.\n",
      "[I 2024-10-06 07:43:52,032] Trial 9 pruned. Trial was pruned at iteration 0.\n",
      "[I 2024-10-06 07:43:52,215] Trial 10 pruned. Trial was pruned at iteration 0.\n",
      "[I 2024-10-06 07:43:52,400] Trial 11 pruned. Trial was pruned at iteration 0.\n",
      "[I 2024-10-06 07:43:52,593] Trial 12 pruned. Trial was pruned at iteration 0.\n",
      "[I 2024-10-06 07:43:52,786] Trial 13 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74292\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:43:52,979] Trial 14 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74288\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:43:53,178] Trial 15 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74288\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:43:53,379] Trial 16 pruned. Trial was pruned at iteration 0.\n",
      "[I 2024-10-06 07:43:53,578] Trial 17 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74297\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:43:53,776] Trial 18 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74289\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:43:53,965] Trial 19 pruned. Trial was pruned at iteration 0.\n",
      "[I 2024-10-06 07:43:54,166] Trial 20 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.73960\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:43:56,303] Trial 21 pruned. Trial was pruned at iteration 929.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.73728\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:43:56,541] Trial 22 pruned. Trial was pruned at iteration 7.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74273\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:43:56,734] Trial 23 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74239\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:43:56,920] Trial 24 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74298\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:43:57,115] Trial 25 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74194\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:43:57,303] Trial 26 pruned. Trial was pruned at iteration 0.\n",
      "[I 2024-10-06 07:43:57,490] Trial 27 pruned. Trial was pruned at iteration 0.\n",
      "[I 2024-10-06 07:43:57,679] Trial 28 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74111\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:43:57,890] Trial 29 pruned. Trial was pruned at iteration 4.\n",
      "[I 2024-10-06 07:43:58,094] Trial 30 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74149\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:43:58,285] Trial 31 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74171\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:43:58,477] Trial 32 pruned. Trial was pruned at iteration 0.\n",
      "[I 2024-10-06 07:43:58,669] Trial 33 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74188\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:43:58,862] Trial 34 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74098\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:43:59,068] Trial 35 pruned. Trial was pruned at iteration 1.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74365\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:43:59,265] Trial 36 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74262\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:43:59,469] Trial 37 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74248\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:43:59,674] Trial 38 pruned. Trial was pruned at iteration 0.\n",
      "[I 2024-10-06 07:43:59,883] Trial 39 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74227\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:00,096] Trial 40 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.73996\n",
      "[459]\tvalidation_0-rmse:0.72706\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:01,561] Trial 41 finished with value: 0.7253787554703454 and parameters: {'learning_rate': 0.3174461686300721, 'reg_lambda': 0.0007524250757900363, 'reg_alpha': 37.597345996584814, 'subsample': 0.5861499298965678, 'colsample_bytree': 0.9433171583811515, 'max_depth': 4, 'min_child_weight': 1, 'gamma': 0.8}. Best is trial 0 with value: 0.7202722001387207.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.73868\n",
      "[337]\tvalidation_0-rmse:0.73487\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:02,866] Trial 42 finished with value: 0.728021337559835 and parameters: {'learning_rate': 0.40016353590807674, 'reg_lambda': 0.0005759860981274814, 'reg_alpha': 34.47060968256257, 'subsample': 0.6043155143604755, 'colsample_bytree': 0.94407707759556, 'max_depth': 6, 'min_child_weight': 1, 'gamma': 0.8}. Best is trial 0 with value: 0.7202722001387207.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.73951\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:03,107] Trial 43 pruned. Trial was pruned at iteration 17.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.73988\n",
      "[343]\tvalidation_0-rmse:0.74326\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:04,249] Trial 44 finished with value: 0.7278519880756795 and parameters: {'learning_rate': 0.3121410149229187, 'reg_lambda': 1.4334664450780358e-05, 'reg_alpha': 1.7793182438540358, 'subsample': 0.6341159503298969, 'colsample_bytree': 0.998173387989596, 'max_depth': 4, 'min_child_weight': 1, 'gamma': 0.9}. Best is trial 0 with value: 0.7202722001387207.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74052\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:04,449] Trial 45 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.73822\n",
      "[335]\tvalidation_0-rmse:0.74432\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:05,485] Trial 46 finished with value: 0.727765381037775 and parameters: {'learning_rate': 0.7327204036655054, 'reg_lambda': 1.4357630814253656e-06, 'reg_alpha': 8.96832953606901, 'subsample': 0.7585611747840432, 'colsample_bytree': 0.9047138733030862, 'max_depth': 3, 'min_child_weight': 2, 'gamma': 0.9}. Best is trial 0 with value: 0.7202722001387207.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.73797\n",
      "[921]\tvalidation_0-rmse:0.72542\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:07,638] Trial 47 finished with value: 0.7260937447980205 and parameters: {'learning_rate': 0.9936232999677419, 'reg_lambda': 3.986052986464263e-07, 'reg_alpha': 95.1895218321526, 'subsample': 0.7512292687413458, 'colsample_bytree': 0.8318396188535166, 'max_depth': 3, 'min_child_weight': 2, 'gamma': 0.9}. Best is trial 0 with value: 0.7202722001387207.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.73841\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:07,854] Trial 48 pruned. Trial was pruned at iteration 10.\n",
      "[I 2024-10-06 07:44:08,046] Trial 49 pruned. Trial was pruned at iteration 0.\n",
      "[I 2024-10-06 07:44:08,229] Trial 50 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.73818\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:08,489] Trial 51 pruned. Trial was pruned at iteration 31.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.73938\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:08,682] Trial 52 pruned. Trial was pruned at iteration 1.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.73894\n",
      "[388]\tvalidation_0-rmse:0.72530\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:09,735] Trial 53 finished with value: 0.725293790426231 and parameters: {'learning_rate': 0.5154499481451604, 'reg_lambda': 1.2084028752655695e-06, 'reg_alpha': 6.176004302576619, 'subsample': 0.988114252782025, 'colsample_bytree': 0.7772350003590761, 'max_depth': 3, 'min_child_weight': 2, 'gamma': 0.9}. Best is trial 0 with value: 0.7202722001387207.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.73998\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:09,949] Trial 54 pruned. Trial was pruned at iteration 0.\n",
      "[I 2024-10-06 07:44:10,168] Trial 55 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74007\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:10,379] Trial 56 pruned. Trial was pruned at iteration 0.\n",
      "[I 2024-10-06 07:44:10,577] Trial 57 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.73952\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:10,777] Trial 58 pruned. Trial was pruned at iteration 1.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74289\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:10,967] Trial 59 pruned. Trial was pruned at iteration 0.\n",
      "[I 2024-10-06 07:44:11,170] Trial 60 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.73796\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:11,412] Trial 61 pruned. Trial was pruned at iteration 18.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.73775\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:11,647] Trial 62 pruned. Trial was pruned at iteration 14.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.73826\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:11,883] Trial 63 pruned. Trial was pruned at iteration 19.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.73710\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:12,115] Trial 64 pruned. Trial was pruned at iteration 14.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.73776\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:12,339] Trial 65 pruned. Trial was pruned at iteration 9.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74299\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:12,533] Trial 66 pruned. Trial was pruned at iteration 0.\n",
      "[I 2024-10-06 07:44:12,730] Trial 67 pruned. Trial was pruned at iteration 0.\n",
      "[I 2024-10-06 07:44:12,924] Trial 68 pruned. Trial was pruned at iteration 0.\n",
      "[I 2024-10-06 07:44:13,121] Trial 69 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74267\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:13,317] Trial 70 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.73982\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:13,514] Trial 71 pruned. Trial was pruned at iteration 2.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.73896\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:14,442] Trial 72 pruned. Trial was pruned at iteration 344.\n",
      "[I 2024-10-06 07:44:14,644] Trial 73 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.73787\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:14,880] Trial 74 pruned. Trial was pruned at iteration 5.\n",
      "[I 2024-10-06 07:44:15,089] Trial 75 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.73983\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:15,304] Trial 76 pruned. Trial was pruned at iteration 3.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74198\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:15,509] Trial 77 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74209\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:15,716] Trial 78 pruned. Trial was pruned at iteration 0.\n",
      "[I 2024-10-06 07:44:15,928] Trial 79 pruned. Trial was pruned at iteration 0.\n",
      "[I 2024-10-06 07:44:16,129] Trial 80 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.73855\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:16,397] Trial 81 pruned. Trial was pruned at iteration 18.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.73970\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:17,736] Trial 82 pruned. Trial was pruned at iteration 344.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.73988\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:18,203] Trial 83 pruned. Trial was pruned at iteration 1.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.73819\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:18,907] Trial 84 pruned. Trial was pruned at iteration 3.\n",
      "[I 2024-10-06 07:44:19,193] Trial 85 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.73936\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:19,387] Trial 86 pruned. Trial was pruned at iteration 2.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74261\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:19,579] Trial 87 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74169\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:19,794] Trial 88 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74014\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:20,011] Trial 89 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74234\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:20,217] Trial 90 pruned. Trial was pruned at iteration 0.\n",
      "[I 2024-10-06 07:44:20,421] Trial 91 pruned. Trial was pruned at iteration 0.\n",
      "[I 2024-10-06 07:44:20,621] Trial 92 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.74038\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:20,831] Trial 93 pruned. Trial was pruned at iteration 0.\n",
      "[I 2024-10-06 07:44:21,026] Trial 94 pruned. Trial was pruned at iteration 0.\n",
      "[I 2024-10-06 07:44:21,222] Trial 95 pruned. Trial was pruned at iteration 0.\n",
      "[I 2024-10-06 07:44:21,417] Trial 96 pruned. Trial was pruned at iteration 0.\n",
      "[I 2024-10-06 07:44:21,614] Trial 97 pruned. Trial was pruned at iteration 0.\n",
      "[I 2024-10-06 07:44:21,818] Trial 98 pruned. Trial was pruned at iteration 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:0.73824\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-10-06 07:44:22,031] Trial 99 pruned. Trial was pruned at iteration 7.\n"
     ]
    }
   ],
   "source": [
    "x, x_val, y, y_val = train_test_split(X_train, y_train, random_state=0, test_size=0.2)\n",
    "x, x_test, y, y_test = train_test_split(x, y, random_state=0, test_size=0.25)\n",
    "\n",
    "study = optuna.create_study(direction=\"minimize\")\n",
    "study.optimize(objective, n_trials=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ef5d856e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-06T07:44:22.076428Z",
     "iopub.status.busy": "2024-10-06T07:44:22.075675Z",
     "iopub.status.idle": "2024-10-06T07:44:22.106860Z",
     "shell.execute_reply": "2024-10-06T07:44:22.105928Z"
    },
    "papermill": {
     "duration": 0.053453,
     "end_time": "2024-10-06T07:44:22.108653",
     "exception": false,
     "start_time": "2024-10-06T07:44:22.055200",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7202722001387207\n",
      "{'learning_rate': 0.029142577621650317, 'reg_lambda': 0.006548888217760044, 'reg_alpha': 8.725393704306269, 'subsample': 0.2637717396252206, 'colsample_bytree': 0.10480925898128796, 'max_depth': 2, 'min_child_weight': 4, 'gamma': 0.1}\n"
     ]
    }
   ],
   "source": [
    "print(study.best_value)\n",
    "print(study.best_params)"
   ]
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "gpu",
   "dataSources": [
    {
     "databundleVersionId": 2267921,
     "sourceId": 27423,
     "sourceType": "competition"
    }
   ],
   "isGpuEnabled": true,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 90.03035,
   "end_time": "2024-10-06T07:44:22.646484",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2024-10-06T07:42:52.616134",
   "version": "2.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
